{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "from torchvision import transforms\n",
    "from torchvision.datasets import CIFAR10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义网络结构"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 残差块 BasicBlock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv3x3(in_channels, out_channels, stride=1):\n",
    "    \"\"\"基本 3*3 卷积层\n",
    "    \"\"\"\n",
    "    return nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv1x1(in_channels, out_channels, stride=1):\n",
    "    \"\"\"基本 1*1 卷积层\n",
    "    \"\"\"\n",
    "    return nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, bias=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class BasicBlock(nn.Module):\n",
    "    \"\"\"残差块,用来转换维度\n",
    "    \"\"\"\n",
    "    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.conv1 = conv3x3(in_channels, out_channels, stride=stride)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.relu = nn.ReLU(True)\n",
    "        \n",
    "        self.conv2 = conv3x3(out_channels, out_channels, stride=1)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "        \n",
    "        self.downsample = downsample\n",
    "        self.stride = stride\n",
    "        \n",
    "        \n",
    "    def forward(self, x):\n",
    "        \"\"\"前向传播\n",
    "        \"\"\"\n",
    "        residual = x\n",
    "        \n",
    "        out = self.conv1(x)\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "        \n",
    "        out = self.conv2(out)\n",
    "        out = self.bn2(out)\n",
    "        \n",
    "        if self.downsample is not None:\n",
    "            residual = self.downsample(residual)\n",
    "            \n",
    "        out += residual\n",
    "        out = self.relu(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 残差网络 ResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class ResNet(nn.Module):\n",
    "    \"\"\"\n",
    "        ResNet 残差网络\n",
    "    \"\"\"\n",
    "    def __init__(self, block, layers, num_classes=10):\n",
    "        \"\"\"\n",
    "        params\n",
    "        ------\n",
    "        block : BasicBlock or Bottleneck(未实现)\n",
    "            残差块的类\n",
    "        layers : List[int]\n",
    "            每一层使用的残差块数\n",
    "        num_classes : int\n",
    "            多分类的类别数\n",
    "        \"\"\"\n",
    "        # cifar10 数据集 32*32*3\n",
    "        super().__init__()\n",
    "        \n",
    "        self.in_channels = 16\n",
    "        # TODO 第一个卷积层的 Kernel_size 选择3 还是 官方的7 ?\n",
    "        # b = 32 * 32 * 3\n",
    "        self.conv1 = nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(16)\n",
    "        self.relu = nn.ReLU(True)\n",
    "        # b = 32 * 32 * 16\n",
    "        self.maxpool = nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        # b = 16 * 16 * 16\n",
    "        self.layer1 = self.make_layer(block, 16, layers[0], stride=1)\n",
    "        # b = 16 * 16 * 16\n",
    "        self.layer2 = self.make_layer(block, 32, layers[1], stride=2)\n",
    "        # b = 8 * 8 * 32\n",
    "        self.layer3 = self.make_layer(block, 64, layers[2], stride=2)\n",
    "        # b = 4 * 4 * 64\n",
    "        self.layer4 = self.make_layer(block, 128, layers[3], stride=2)\n",
    "        \n",
    "        # 全局平均池化\n",
    "        # b = 2 * 2 * 128\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n",
    "        # b = 1 * 1 * 128\n",
    "        self.fc = nn.Linear(128, num_classes)\n",
    "        \n",
    "        # 初始化权重（卷积层无偏置）\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                nn.init.kaiming_normal_(m.weight, mode=\"fan_out\", nonlinearity=\"relu\")\n",
    "            elif isinstance(m, nn.BatchNorm2d):\n",
    "                nn.init.constant_(m.weight, 1)\n",
    "                nn.init.constant_(m.bias, 0)\n",
    "            \n",
    "        \n",
    "    def make_layer(self, block, out_channels, blocks, stride=1):\n",
    "        \"\"\"\n",
    "        动态加入层\n",
    "        \n",
    "        params\n",
    "        ------\n",
    "        block : BasicBlock or Bottleneck(未实现)\n",
    "        out_channels : int\n",
    "            输出通道数\n",
    "        blocks : int\n",
    "            本层使用的残差快数量\n",
    "        stride : int \n",
    "            滑动步长\n",
    "        \"\"\"\n",
    "        downsample = None\n",
    "        # BasicBlock 无 expansion, 直接比较 in_channels 和 out_channels 即可\n",
    "        if stride != 1 or self.in_channels != out_channels:\n",
    "            downsample = nn.Sequential(\n",
    "                # TODO 输出大小?\n",
    "                conv1x1(self.in_channels, out_channels, stride=stride),\n",
    "                nn.BatchNorm2d(out_channels)\n",
    "            )\n",
    "            \n",
    "        layers = []\n",
    "        layers.append(block(self.in_channels, out_channels, stride, downsample))\n",
    "        \n",
    "        self.in_channels = out_channels\n",
    "        \n",
    "        for _ in range(1, blocks):\n",
    "            layers.append(block(self.in_channels, out_channels))\n",
    "            \n",
    "        return nn.Sequential(*layers)\n",
    "            \n",
    "        \n",
    "    def forward(self, x):\n",
    "        \"\"\"前向传播\n",
    "        \"\"\"\n",
    "        # 长度 * 宽度 * 通道数\n",
    "        # b = 32 * 32 * 3\n",
    "        out = self.conv1(x)\n",
    "        # b = 32 * 32 * 16\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.maxpool(out)\n",
    "        # b = 16 * 16 * 16\n",
    "        out = self.layer1(out)\n",
    "        # b = 16 * 16 * 16\n",
    "        out = self.layer2(out)\n",
    "        # b = 8 * 8 * 32\n",
    "        out = self.layer3(out)\n",
    "        # b = 4 * 4 * 64\n",
    "        out = self.layer4(out)\n",
    "        # b = 2 * 2 * 128\n",
    "        out = self.avgpool(out)\n",
    "        # b = 1 * 1 * 128\n",
    "        # 展平, 输入全连接层\n",
    "        # out = torch.flatten(out, 1)\n",
    "        out = out.squeeze()\n",
    "        # b = 128\n",
    "        out = self.fc(out)\n",
    "        # b = 10\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读入数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练集数据增强"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_transform = transforms.Compose(\n",
    "    [\n",
    "        # 以0.5的概率随机水平翻转\n",
    "        transforms.RandomHorizontalFlip(0.5),\n",
    "        # 四周填充4个像素后裁剪为 32*32\n",
    "        transforms.RandomCrop(32, padding=4),\n",
    "        transforms.ToTensor(),\n",
    "        # 像素点值缩放至 [-1, 1]\n",
    "        transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 测试集标准化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_transform = transforms.Compose(\n",
    "    [\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 读入数据集, 划分训练集和验证集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train: 40000   val: 10000  test: 10000\n"
     ]
    }
   ],
   "source": [
    "train_db = CIFAR10(root=\"./data\", train=True, transform=train_transform, download=False)\n",
    "test_db = CIFAR10(root=\"./data\", train=False, transform=test_transform, download=False)\n",
    "\n",
    "# 划分训练集和验证集, 比例 4:1\n",
    "train_db, val_db = torch.utils.data.random_split(train_db, [int(len(train_db)*0.8), int(len(train_db)*0.2)])\n",
    "\n",
    "print(f\"train: {len(train_db)}   val: {len(val_db)}  test: {len(test_db)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义超参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 256\n",
    "# 训练的总轮次, 由于early_stopping 存在, 可能提前结束\n",
    "NUM_EPOCH = 20\n",
    "# 学习率\n",
    "LR = 0.001\n",
    "# 读取数据集时的进程数\n",
    "WORKERS = 4\n",
    "\n",
    "\n",
    "# 10个epoch 内表现不再提升则 early stop\n",
    "PATIENCE = 5\n",
    "# early_stopping 监控指标\n",
    "monitor = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 损失函数与优化器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 先将模型放在GPU上, 再创建优化器\n",
    "model = ResNet(BasicBlock, [2, 2, 2, 2], num_classes=10).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 多分类损失函数 交叉熵\n",
    "criterion = nn.CrossEntropyLoss().cuda()\n",
    "# Adam 优化器, L2 正则化项权重 0.01\n",
    "optimizer = optim.Adam(model.parameters(), lr=LR, weight_decay=0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def checkpoint(epoch, model, optimizer, val_loss, val_acc, file_path=\"./model/checkpoint.pth\"):\n",
    "    \"\"\" 保存模型checkpoint\n",
    "    \"\"\"\n",
    "    state = {\n",
    "        'epoch': epoch,\n",
    "        'model': model.state_dict(),\n",
    "        'optimizer': optimizer.state_dict(),\n",
    "        'val_loss': val_loss,\n",
    "        'val_acc': val_acc\n",
    "    }\n",
    "    \n",
    "    torch.save(state, file_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_db, batch_size=BATCH_SIZE, shuffle=True, num_workers=WORKERS, pin_memory=True)\n",
    "val_loader = DataLoader(val_db, batch_size=BATCH_SIZE, shuffle=True, num_workers=WORKERS, pin_memory=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_acc_list = []\n",
    "train_loss_list = []\n",
    "val_acc_list = []\n",
    "val_loss_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/20  loss: 0.006663  acc: 0.370450  val_loss: 0.006448  val_acc: 0.428100\n",
      "1/20  => saving checkpoint with  val_loss:0.006448  val_acc:0.428100\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-51ff34dd2d60>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     39\u001b[0m     \u001b[0mcorrect\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0.0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 41\u001b[1;33m     \u001b[1;32mfor\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mval_loader\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     42\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_available\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     43\u001b[0m             \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mVariable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\torch\\lib\\site-packages\\torch\\utils\\data\\dataloader.py\u001b[0m in \u001b[0;36m__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    276\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0m_SingleProcessDataLoaderIter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    277\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 278\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0m_MultiProcessingDataLoaderIter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    279\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    280\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\torch\\lib\\site-packages\\torch\\utils\\data\\dataloader.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, loader)\u001b[0m\n\u001b[0;32m    680\u001b[0m             \u001b[1;31m#     before it starts, and __del__ tries to join but will get:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    681\u001b[0m             \u001b[1;31m#     AssertionError: can only join a started process.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 682\u001b[1;33m             \u001b[0mw\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    683\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex_queues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex_queue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    684\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mworkers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\torch\\lib\\multiprocessing\\process.py\u001b[0m in \u001b[0;36mstart\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    103\u001b[0m                \u001b[1;34m'daemonic processes are not allowed to have children'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    104\u001b[0m         \u001b[0m_cleanup\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 105\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_popen\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_Popen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    106\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sentinel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_popen\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msentinel\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    107\u001b[0m         \u001b[1;31m# Avoid a refcycle if the target function holds an indirect\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\torch\\lib\\multiprocessing\\context.py\u001b[0m in \u001b[0;36m_Popen\u001b[1;34m(process_obj)\u001b[0m\n\u001b[0;32m    221\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    222\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_Popen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 223\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_default_context\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_context\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mProcess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_Popen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    224\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    225\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mDefaultContext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBaseContext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\torch\\lib\\multiprocessing\\context.py\u001b[0m in \u001b[0;36m_Popen\u001b[1;34m(process_obj)\u001b[0m\n\u001b[0;32m    320\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0m_Popen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    321\u001b[0m             \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mpopen_spawn_win32\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mPopen\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 322\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mPopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    323\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    324\u001b[0m     \u001b[1;32mclass\u001b[0m \u001b[0mSpawnContext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBaseContext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\torch\\lib\\multiprocessing\\popen_spawn_win32.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, process_obj)\u001b[0m\n\u001b[0;32m     63\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     64\u001b[0m                 \u001b[0mreduction\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprep_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mto_child\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 65\u001b[1;33m                 \u001b[0mreduction\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mto_child\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     66\u001b[0m             \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     67\u001b[0m                 \u001b[0mset_spawning_popen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\torch\\lib\\multiprocessing\\reduction.py\u001b[0m in \u001b[0;36mdump\u001b[1;34m(obj, file, protocol)\u001b[0m\n\u001b[0;32m     58\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m     \u001b[1;34m'''Replacement for pickle.dump() using ForkingPickler.'''\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 60\u001b[1;33m     \u001b[0mForkingPickler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     61\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m \u001b[1;31m#\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\torch\\lib\\multiprocessing\\queues.py\u001b[0m in \u001b[0;36m__getstate__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     56\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     57\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__getstate__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 58\u001b[1;33m         \u001b[0mcontext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0massert_spawning\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     59\u001b[0m         return (self._ignore_epipe, self._maxsize, self._reader, self._writer,\n\u001b[0;32m     60\u001b[0m                 self._rlock, self._wlock, self._sem, self._opid)\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "best_val_acc = 0.0\n",
    "best_val_loss = float('inf')\n",
    "\n",
    "# for early_stopping\n",
    "monitor = float('inf')\n",
    "epoch_left = PATIENCE\n",
    "# down = True\n",
    "\n",
    "\n",
    "for epoch in range(NUM_EPOCH):\n",
    "    # 训练\n",
    "    model.train()\n",
    "    correct = 0.0\n",
    "    train_loss = 0.0\n",
    "    val_loss = 0.0\n",
    "        \n",
    "    for X, y in train_loader:\n",
    "        if torch.cuda.is_available():\n",
    "            X, y = Variable(X).cuda(), Variable(y).cuda()\n",
    "        else:\n",
    "            X, y = Variable(X), Variable(y)\n",
    "            \n",
    "        optimizer.zero_grad()\n",
    "        out = model(X)\n",
    "        loss = criterion(out, y)\n",
    "        train_loss += loss.item()\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        prediction = out.data.max(1)[1]\n",
    "        correct += float(prediction.eq(y.data).sum().data)\n",
    "        \n",
    "    train_acc = correct / len(train_loader.dataset)\n",
    "    train_loss /= len(train_loader.dataset)\n",
    "    \n",
    "    # 验证集上验证\n",
    "    model.eval()\n",
    "    correct = 0.0\n",
    "    \n",
    "    for X, y in val_loader:\n",
    "        if torch.cuda.is_available():\n",
    "            X, y = Variable(X).cuda(), Variable(y).cuda()\n",
    "        else:\n",
    "            X, y = Variable(X), Variable(y)\n",
    "            \n",
    "        out = model(X)\n",
    "        loss = criterion(out, y)\n",
    "        val_loss += loss.item()\n",
    "        \n",
    "        prediction = out.data.max(1)[1]\n",
    "        correct += float(prediction.eq(y.data).sum().data)\n",
    "        \n",
    "    val_acc = correct / len(val_loader.dataset)\n",
    "    val_loss /= len(val_loader.dataset)\n",
    "    \n",
    "    train_acc_list.append(train_acc)\n",
    "    train_loss_list.append(train_loss)\n",
    "    val_acc_list.append(val_acc)\n",
    "    val_loss_list.append(val_loss)\n",
    "    \n",
    "    print(f\"{epoch+1}/{NUM_EPOCH}  loss: {train_loss:2.6f}  acc: {train_acc:1.6f}  val_loss: {val_loss:2.6f}  val_acc: {val_acc:1.6f}\")\n",
    "    \n",
    "    # 是否当前最优\n",
    "    if (val_acc > best_val_acc) and (val_loss <= best_val_loss):\n",
    "        checkpoint(epoch+1, model, optimizer, val_loss, val_acc, \"./model/resnet_test_checkpoint.pth\")\n",
    "        print(f\"{epoch+1}/{NUM_EPOCH}  => saving checkpoint with  val_loss:{val_loss:2.6f}  val_acc:{val_acc:1.6f}\")\n",
    "        best_val_acc = val_acc\n",
    "        best_val_loss = val_loss\n",
    "        monitor = val_loss\n",
    "        epoch_left = PATIENCE\n",
    "        \n",
    "    elif val_loss > monitor:\n",
    "        epoch_left -= 1\n",
    "        if epoch_left == 0:\n",
    "            break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 绘制训练曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl81OW59/HPJbKjyGZVtlCLG0gA\nI7jgggviBu6CtBUVcSlq1Z4Wq6dyUHxaF+RoqRUt6qlR9NGjggu2Vi3yUCBBBQSlLAJGEAMiokEk\ncD1/3JMwCZNkApP8Jpnv+/XKK/Pbrxn0mjv37/5dt7k7IiKSGfaKOgAREak9SvoiIhlESV9EJIMo\n6YuIZBAlfRGRDKKkLyKSQZT0ZbeYWQMz+9bMOqVy3yiZ2U/MLOVjmM3sNDNbGbe8xMxOSGbf3bjW\n42b22909Xuq/vaMOQGqHmX0bt9gM2Apsjy1f4+651Tmfu28HWqR630zg7oem4jxmNgL4qbufHHfu\nEak4t9RfSvoZwt1Lk26sJTnC3d+qaH8z29vdi2sjNhGpPereEQDM7G4ze87MnjWzzcBPzexYM5tt\nZl+b2Voze8jMGsb239vM3MyyYstPx7a/YWabzexfZtaluvvGtp9pZv82s01m9rCZ/T8zG15B3MnE\neI2ZLTOzjWb2UNyxDczsQTPbYGbLgYGVfD53mNmUcusmmtn42OsRZvZx7P0sj7XCKzpXgZmdHHvd\nzMz+GottEXBUguuuiJ13kZkNiq0/EvgjcEKs62x93Gc7Ju74a2PvfYOZvWxmBybz2VTncy6Jx8ze\nMrOvzOwLM/t13HX+M/aZfGNm+WZ2UEXXkVrg7vrJsB9gJXBauXV3Az8A5xIaA02Bo4G+hL8Ifwz8\nGxgV239vwIGs2PLTwHogB2gIPAc8vRv77g9sBgbHtt0CbAOGV/BekonxFaAlkAV8VfLegVHAIqAD\n0AaYEf6XSHidHwPfAs3jzv0lkBNbPje2jwGnAFuAHrFtpwEr485VAJwce30/8C7QCugMLC637yXA\ngbF/k8tiMfwotm0E8G65OJ8GxsReD4jF2BNoAvwJeDuZz6aan3NLYB1wE9AY2BfoE9t2GzAf6Bp7\nDz2B1lH/P5DJP2rpS7yZ7j7N3Xe4+xZ3z3P3Oe5e7O4rgEnASZUc/4K757v7NiCX8D94dfc9B/jQ\n3V+JbXuQ8AWRUJIx/h933+TuKwkJtuRalwAPunuBu28Afl/JdVYAHxG+jABOB7529/zY9mnuvsKD\nt4F/AAlv1pZzCXC3u29091WE1nv8dZ9397Wxf5NnCF/YOUmcF2AY8Li7f+ju3wOjgZPMrEPcPhV9\nNmVU8TkPAj5z9/92963u/o27z41tGwH81t2Xxt7Dh+7+VZLxSw1Q0pd4n8UvmNlhZvZa7M/1b4Cx\nQNtKjv8i7nURld+8rWjfg+LjcHcntIwTSjLGpK4FrKokXoBngKGx15cRvqxK4jjHzObEuje+JrSy\nK/usShxYWQxmNtzM5se6Vb4GDkvyvBDeX+n53P0bYCPQPm6fpP7NqvicOwLLKoihI7A8yXilFijp\nS7zywxUfJbRuf+Lu+wK/I3Rf1KS1hO4WAMzMKJukytuTGNcSklKJqoaUPgecFmspDyZ8CWBmTYEX\ngP9D6HrZD/hbknF8UVEMZvZj4BHgOqBN7LyfxJ23quGlawhdRiXn24fQjfR5EnGVV9nn/BlwcAXH\nVbZNIqCkL5XZB9gEfGdmhwPX1MI1XwV6m9m5ZrY3oZ+4XQ3F+DzwSzNrb2ZtgN9UtrO7rwNmAk8A\nS9x9aWxTY6ARUAhsN7NzgFOrEcNvzWw/C88xjIrb1oKQ2AsJ338jCC39EuuADvE3VMt5FrjKzHqY\nWWPCl9J77l7hX06VqOxzngp0MrNRZtbIzPY1sz6xbY8Dd5vZwRb0NLPWu3F9SRElfanMrcDlhBur\njxJaujUqllgvBcYDGwitxA8IzxWkOsZHCH3vC4E8Qmu9Ks8Qbsw+Exfz18DNwEuEm6EXEb68knEn\n4S+OlcAbwP/EnXcB8BAwN7bPYcCcuGP/DiwF1plZfDdNyfHTCd0wL8WO70To598dFX7O7r6JcI/j\nQsKN43+zs7//PuBlwuf8DeFeQJPdjEFSwEKXqUh6MrMGhG6Ki9z9vajjEanr1NKXtGNmA82sZaxL\n4j+BYkJrV0T2kJK+pKN+wArCUM2BwHnuXlH3johUg7p3REQyiFr6IiIZJO0KrrVt29azsrKiDkNE\npE6ZN2/eenevbHgzkIZJPysri/z8/KjDEBGpU8ysqifKAXXviIhkFCV9EZEMklTSj42bXhKruz26\ngn0uMbPFsZrfz8St325mH8Z+pqYqcBERqb4q+/RjT0ROJDxmXQDkmdlUd18ct09XQt3s4919o5nt\nH3eKLe5eWYndKm3bto2CggK+//77PTmN1LAmTZrQoUMHGjasqBSMiEQtmRu5fYBlsRraxGYPGkyY\n7KHE1cBEd98I4O5fpjLIgoIC9tlnH7KysghFFyXduDsbNmygoKCALl26VH2AiEQime6d9pSt913A\nrqVuDwEOsTCt3Wwzi592rklsirTZZnZeoguY2cjYPvmFhYW7bP/+++9p06aNEn4aMzPatGmjv8ZE\ndkNuLmRlwV57hd+5uVUdsfuSaeknyrTlH+PdmzAd2smEWujvmVn3WPXBTu6+JlYb/G0zW+juZSZV\ncPdJhOp75OTkJHxEWAk//enfSKT6cnNh5EgoKgrLq1aFZYBhu1sTtRLJtPQLKDvJQwdC1cPy+7zi\n7tvc/VNgCeFLAHdfE/u9gjAdW689jFlEpN64/fadCb9EUVFYXxOSSfp5QFcz62JmjYAhhEkT4r0M\n9Acws7aE7p4VZtYqVimxZP3xlL0XUCds2LCBnj170rNnTw444ADat29fuvzDDz+U2feMM85g8+bN\nu3WdiRMnkluTf9eJSNpZvbp66/dUld077l5sZqOAN4EGwGR3X2RmY4F8d58a2zbAzBYD24H/cPcN\nZnYc8KiZ7SB8wfw+ftRPTcnNDd+Sq1dDp04wbtye/ZnUpk0bPvzwQwDGjBlDixYt+NWvflVmn5KZ\n5t98883dvs4vfvGL3Q9SROqkTp1Cl06i9TUhqXH67v66ux/i7ge7+7jYut/FEj4e3OLuR7j7ke4+\nJbZ+Vmw5O/b7LzXzNnYq6R9btQrcd/aP1UQDetmyZXTv3p1rr72W3r17s3btWjp06MDXX38NwLnn\nnstRRx1Ft27dePzxxwEoLi5mv/32Y/To0WRnZ3Psscfy5ZdhsNMdd9zBhAkTAOjXrx+jR4+mT58+\nHHroocyaNQuA7777jgsvvJDs7GyGDh1KTk5O6RdSvDvvvJOjjz66NL6Saqr//ve/OeWUU8jOzqZ3\n796sXLkSgHvuuYcjjzyS7Oxsbq+pvytFZBfjxkGzZmXXNWsW1teEevdEbm33jy1evJirrrqKDz74\ngPbtyw5qeuqpp5g3bx55eXmMHz+ejRs3ArBp0yZOOukk5s+fz7HHHsvkyZMTntvdmTt3Lvfddx9j\nx44F4OGHH+aAAw5g/vz5jB49mg8++CDhsTfddBN5eXksXLiQTZs2MX36dACGDh3KzTffzPz585k1\naxb7778/06ZN44033mDu3LnMnz+fW2+9NVUfj4hUYdgwmDQJOncGs/B70qSauYkL9TDp13b/2MEH\nH8zRRx+dcNuDDz5Y2povKChg+fIwaKlp06aceeaZABx11FGlre3yLrjggl32mTlzJkOGDAEgOzub\nbt26JTz2H//4B3369CE7O5t//vOfLFq0iI0bN7J+/XrOPfdcIDxM1axZM9566y2uvPJKmjZtCkDr\n1pq3WqQ2DRsGK1fCjh3hd00lfEjDKpt7qrb7x5o3b55w/VtvvcWMGTOYPXs2TZs2pV+/fqVj2Bs1\nalS6X4MGDSguLk54jsaNG++yTzKT3hQVFTFq1Cjef/992rdvzx133FF67UTDKt1dwy1FMkS9a+nX\ndv9YRTZt2kTr1q1p2rQpixYtIi8vLyXn7devH88//zwACxcuZPHiXe+Lb9myhb322ou2bduyefNm\nXnzxRQBatWpF27ZtmTZtGhAeeisqKmLAgAH85S9/YcuWLQB89dVXKYlVRNJPvUv6td0/VpGzzz6b\noqIisrOzGTt2LH379k3JeW+44QY+//xzevTowQMPPED37t1p2bJlmX3atGnD5ZdfTvfu3Tn//PPL\nXDs3N5cHHniAHj160K9fPwoLCznnnHMYOHAgOTk59OzZkwcffDAlsYqku9p8EjZdpN0cuTk5OV5+\nEpWPP/6Yww8/PKKI0ktxcTHFxcU0adKEpUuXMmDAAJYuXcree6dHT53+raSuKP8kLIRegSgaialg\nZvPcPaeq/dIjU0jSvv32W0499VSKi4txdx599NG0SfgidUllI/3qYtJPlrJFHbPffvsxb968qMMQ\nqfNqe6RfVRYsgHXr4PTTa/Y69a5PX0QkGRWN6KupkX6JbNgADz8MRx0F2dlw0001f00lfRHJSFGN\n9Csuhtdfh4svhoMOghtvDNUDHnoI3nuvZq8N6t4RkQxV0m+fyjpdlfnkE3jiCfjrX2HtWmjbFq6/\nHoYPD6382qKkLyIZa9iwmr1pu2kTTJkCTz4Js2dDgwZw1llwxRVw9tkQ95xmrVH3ThJOPvnkXapn\nTpgwgeuvv77S41q0aAHAmjVruOiiiyo8d/khquVNmDCBorhhBmeddVZpUTcRSS87dsBbb4UvkwMO\ngGuvhc2b4f774fPPYepUOP/8aBI+KOknZejQoUyZMqXMuilTpjB06NCkjj/ooIN44YUXdvv65ZP+\n66+/zn777bfb5xOR1Fu+HP7zP8NDXqefHvrtr7gC5s6FhQvh1lvhRz+KOkol/aRcdNFFvPrqq2zd\nuhWAlStXsmbNGvr161c6br53794ceeSRvPLKK7scv3LlSrp37w6EEglDhgyhR48eXHrppaWlDwCu\nu+46cnJy6NatG3feeScADz30EGvWrKF///70798fgKysLNavXw/A+PHj6d69O927dy8ty7xy5UoO\nP/xwrr76arp168aAAQPKXKfEtGnT6Nu3L7169eK0005j3bp1QHgW4IorruDII4+kR48epWUcpk+f\nTu/evcnOzubUU09NyWcrmam+PAn77behn/6kk+AnPwn3BI44InTprF0Lf/oTHH10qA6QLupcn/4v\nfwkJysfvkZ49IZYvE2rTpg19+vRh+vTpDB48mClTpnDppZdiZjRp0oSXXnqJfffdl/Xr13PMMccw\naNCgCguYPfLIIzRr1owFCxawYMECevfuXbpt3LhxtG7dmu3bt3PqqaeyYMECbrzxRsaPH88777xD\n27Zty5xr3rx5PPHEE8yZMwd3p2/fvpx00km0atWKpUuX8uyzz/LYY49xySWX8OKLL/LTn/60zPH9\n+vVj9uzZmBmPP/449957Lw888AB33XUXLVu2ZOHChQBs3LiRwsJCrr76ambMmEGXLl1Un0d2W23P\nCZtq7mGUzRNPwP/9v/Ddd9C1K9xzD/zsZ9ChQ9QRVk4t/STFd/HEd+24O7/97W/p0aMHp512Gp9/\n/nlpizmRGTNmlCbfHj160KNHj9Jtzz//PL1796ZXr14sWrQoYTG1eDNnzuT888+nefPmtGjRggsu\nuID3YmO+unTpQs+ePYGKyzcXFBRwxhlncOSRR3LfffexaNEiIFQIjZ/Fq1WrVsyePZsTTzyRLl26\nACq/LLuvtue8SJXVq+Guu0KL/qST4IUXYMgQmDkTliyB225L/4QPdbClX1mLvCadd9553HLLLbz/\n/vts2bKltIWem5tLYWEh8+bNo2HDhmRlZZWWMa5Ior8CPv30U+6//37y8vJo1aoVw4cPr/I8ldVN\nKinLDKE0c6LunRtuuIFbbrmFQYMG8e677zJmzJjS85aPUeWXJVXS7UnYymzZAi+9FFr1//hHaOX3\n7w9jxsAFF0AFldXTmlr6SWrRogUnn3wyV155ZZkbuJs2bWL//fenYcOGvPPOO6xKVMw/zoknnlg6\n+flHH33EggULAPjmm29o3rw5LVu2ZN26dbzxxhulx+yzzz4JJ1s/8cQTefnllykqKuK7777jpZde\n4oQTTkj6PW3atKl0tq+nnnqqdP2AAQP44x//WLq8ceNGjj32WP75z3/y6aefAiq/LLsvHZ6ErYx7\nGF55zTVh9M2wYbB0Kfzud7BiBbz9dujGqYsJH5T0q2Xo0KHMnz+/dOYqgGHDhpGfn09OTg65ubkc\ndthhlZ7juuuu49tvv6VHjx7ce++99OnTBwizYPXq1Ytu3bpx5ZVXcvzxx5ceM3LkSM4888zSG7kl\nevfuzfDhw+nTpw99+/ZlxIgR9OrVK+n3M2bMGC6++GJOOOGEMvcL7rjjDjZu3Ej37t3Jzs7mnXfe\noV27dkyaNIkLLriA7OxsLr300qSvIxIvXea8KG/NGvjDH8KN2GOPDQ9RDR4ckvyKFaF1H+vdrNNU\nWllSSv9Wkozc3Np7EraEe3hY6ssvobAw/C75mT0bpk8PY+yPOy4MtbzkEth335qNKZVSWlrZzAYC\n/w00AB53998n2OcSYAzgwHx3vyy2/nLgjthud7v7U+WPFZHMkqonYYuKyibvRAk9fnnbtsTn6dgR\nfvObUBLhkEP2PK50VmXSN7MGwETgdKAAyDOzqe6+OG6frsBtwPHuvtHM9o+tbw3cCeQQvgzmxY7d\nmPq3IiJ13Q8/7EzSlSXvkp/yo4BKNGsG++8fftq3D8OyS5b33x/atdv5um1biBv3UO8l09LvAyxz\n9xUAZjYFGAzEjye8GphYkszd/cvY+jOAv7v7V7Fj/w4MBJ6tbqAaPZL+0q2rUNLP+vWh8NiSJeFn\n+fJQQ74koVdUXaRhw7IJ+5BDEifwkuW6epO1NiST9NsDn8UtFwDlJ3w9BMDM/h+hC2iMu0+v4Nj2\n5S9gZiOBkQCdEtzCb9KkCRs2bKBNmzZK/GnK3dmwYQNNmjSJOhSJ2LZtIZkvWVI2wX/yCcQP+mrU\nCA4+OIyQ6dWr8iTesmV6PdValyWT9BN91OWbdHsDXYGTgQ7Ae2bWPcljcfdJwCQIN3LLb+/QoQMF\nBQUUFhYmEa5EpUmTJnSoC0+nSErEt9rLt963b9+53wEHwGGHhfrxhx4aXh96KHTuHKpOSu1KJukX\nAB3jljsAaxLsM9vdtwGfmtkSwpdAAeGLIP7Yd6sbZMOGDUufBBWRPVOdkTM//BCGK5ZvsS9ZUrbV\n3rhxKEVw5JFlk/shh4RWuqSPZJJ+HtDVzLoAnwNDgMvK7fMyMBR40szaErp7VgDLgXvMrFVsvwGE\nG74iEoFEdW+uvhq++SYk7Pik/sknIeHHt9oPPDAk9Isv3tliV6u9bqky6bt7sZmNAt4k9NdPdvdF\nZjYWyHf3qbFtA8xsMbAd+A933wBgZncRvjgAxpbc1BWR2peo7s2WLWEGpxIlrfbs7DBWXa32+qVO\nPJwlInuuuDiMgqnI66+H5N6pk1rtdVGyD2epDINIPecOb7wBcQVdd9G5M5x5ZigzoIRfvynpi9Rj\nCxfCwIFhXtbiYrj5ZmjatOw+6VD3RmqPkr5IPbRuXagS2bMn5OWFkuQffQTjx8Njj4WWvVn4PWlS\n3Zi8RFKjztXTF5GKbdkSEvw998D338ONN4Z5W+PnvElV3Rupm5T0ReoB9zAv6+jRYfz9eeeFMsH1\nvXiYVJ+6d0TquFmzQv33yy6DNm3gnXfCbE9K+JKIkr5IHfXpp3DppXD88fDZZ/Dkk5CfDyefHHVk\nks7UvSNSx2zaFPrsJ0yAvfcOMzr96leqLCnJUdIXqSOKi8PIm9/9DjZsgMsvh7vvDvXiRZKl7h2R\nWpKbC1lZsNde4XdubvLHvvFGKItw/fXQrVvoxnniCSV8qT4lfZFaUFLobNWqMNJm1aqwXFXi/+ij\nnQ9X/fADvPxyuFHbu3ftxC31j5K+SC1IVOisqCisT2TdOrj22tC6nzs39N8vWgSDB2syEdkz6tMX\nqQWrVye3/vvvdz5ctWUL3HBD6MOPf7hKZE+opS9SCxLMAlpmfcnDVYcdBrfdBqecElr2EyYo4Utq\nKemL1IJx40Jhs3glhc7+9S847jgYOhRatYK33w5993q4SmqCkr5ILRg2LBQ2iy90Nm4cTJsWEv6q\nVWE0Tn4+9O8fdbRSnynpi9SSYcNg5UrYuBGGDAl1cqZOhTvvhKVLYfhw1bKXmqcbuSI1bMuWkOxX\nroQFC+CBB6CwMDxcNW6cxtpL7VLSF9lD27aF2jeffhoS+6eflv354ouy+594YnjY6qijIglXMpyS\nvkgVduyAtWsrTuoFBbB9+879GzSAjh3D1INnnRV+l/xkZcGBB2qsvURHSV/qvdzc8BDU6tVhiOS4\ncWUnEXEPtWwqSuqrVsHWrWXPeeCBIYn361c2qXfpAh06hEJoIulI/2lKvVZS/qDkadhVq+CKK+Cv\nf4XGjXcm9m+/LXtc69YhgffoEZ6CjU/qnTtDkya1/15EUiGppG9mA4H/BhoAj7v778ttHw7cB3we\nW/VHd388tm07sDC2frW7D0pB3CJJSVT+YNs2+NvfQuGyLl3CEMnyXTD77htJuCI1rsqkb2YNgInA\n6UABkGdmU919cbldn3P3UQlOscXde+55qCLV88UXoWWfiDssXJh4m0h9lsw4/T7AMndf4e4/AFOA\nwTUblsjuc4fJk+Hwwyvep3Pn2otHJJ0kk/TbA5/FLRfE1pV3oZktMLMXzKxj3PomZpZvZrPN7LxE\nFzCzkbF98gsLC5OPXqScFSvg9NPhqqtCf/x991Vc/kAkEyWT9BMNLvNyy9OALHfvAbwFPBW3rZO7\n5wCXARPM7OBdTuY+yd1z3D2nXbt2SYYuslNxcXjoqXt3yMuDP/851J3/1a92LX8waVLZ0TsimSSZ\nG7kFQHzLvQOwJn4Hd98Qt/gY8Ie4bWtiv1eY2btAL2D5bsYrsov582HEiFC3ZtAg+NOfyj7lOmyY\nkrxIiWRa+nlAVzPrYmaNgCHA1PgdzOzAuMVBwMex9a3MrHHsdVvgeKD8DWCR3fL992F0Tk5OGIP/\n3HOhOqXKGohUrMqWvrsXm9ko4E3CkM3J7r7IzMYC+e4+FbjRzAYBxcBXwPDY4YcDj5rZDsIXzO8T\njPoRqbb33oOrr4YlS0KhsvvvhzZtoo5KJP2Ze/nu+Wjl5OR4fn5+1GFImvrmG/jNb0KffVZW6J8/\n/fSooxKJnpnNi90/rZRKK0udMW0aHHFESPQ33xwmDVfCF6keJX1Je19+GerPDxoUZpb6179g/Hho\n3jzqyETqHiV9SVvu8D//Ex6yeuklGDsW5s2DPn2ijkyk7lLBNUlLK1fCNdeEGjnHHQePPRa6dkRk\nz6ilL2ll+3aYMCEUQ5s1Cx5+OIzUUcIXSQ219CVtfPRReMhqzhw488wwQqdTp6ijEqlf1NKXGpOb\nG4ZV7rVX+J2bm3i/rVvD5OC9e8Py5WG/115TwhepCWrpS41INHnJyJHhdXxJhFmzQuv+44/D+gcf\nBJVfEqk5aulLjUg0eUlRUVgPsHkz3HBDmG7w22/h9dfh6aeV8EVqmpK+1IjVqyte/8Yb4UbtxIkw\nahQsWhT68EWk5ql7R2pEp06JZ61q1gzOOiuMvZ85MwzHFJHao5a+1Ihx43advARCZczf/Q4++EAJ\nXyQKaulLjSi5Wfub38Dnn4fXBx8cSh937x5dXCKZTi19qREbN4bJTdavDy3+Bx8MZZCV8EWipZa+\npNTWreEG7d13w9dfw89+BnfdpTH3IulCLX1JiR074Nln4bDD4NZbw2xW778PTz2lhC+STpT0ZY+9\n+y707QuXXQb77gtvvhkKpfXsGXVkIlKekr7stkWL4JxzoH9/+OILePLJ0LofMCDqyESkIkr6Um1r\n1oT5aXv0CBUwf/97+Pe/4fLLoUGDqKMTkcroRq4kbfNmuPdeeOABKC6GG28MZRXato06MhFJlpK+\nVGnbtjCJyZgxUFgIl14aHr46+OCoIxOR6kqqe8fMBprZEjNbZmajE2wfbmaFZvZh7GdE3LbLzWxp\n7OfyVAYvNcs9TFPYvTv84hehdMKcOTBlihK+SF1VZUvfzBoAE4HTgQIgz8ymuvvicrs+5+6jyh3b\nGrgTyAEcmBc7dmNKopcaM2sW/Md/hN+HHw5Tp4abtmZRRyYieyKZln4fYJm7r3D3H4ApwOAkz38G\n8Hd3/yqW6P8ODNy9UKU2LF0KF10Exx8PK1bAo4/CggVw7rlK+CL1QTJJvz3wWdxyQWxdeRea2QIz\ne8HMOlbzWInYl1+GMsdHHAHTp8N//Vf4Ahg5EvbWnR+ReiOZpJ+ofefllqcBWe7eA3gLeKoax2Jm\nI80s38zyCwsLkwhJUqWoKNyU/clPwpy0I0bAsmWhEmaLFlFHJyKplkzSLwA6xi13ANbE7+DuG9x9\na2zxMeCoZI+NHT/J3XPcPaedpk6qFdu3w+TJ0LUr3HEHnHJKmJj8kUfggAOijk5EakoyST8P6Gpm\nXcysETAEmBq/g5kdGLc4CPg49vpNYICZtTKzVsCA2DqJiHuYmrBnT7jqKujYEWbMCCWPDzss6uhE\npKZV2Vvr7sVmNoqQrBsAk919kZmNBfLdfSpwo5kNAoqBr4DhsWO/MrO7CF8cAGPd/asaeB+ShHnz\n4Ne/hrffDkMun38+3LTVDVqRzGHuu3SxRyonJ8fz8/OjDqNOy80NT8quXh0qXP7yl5CXB888A23a\nwJ13wjXXQKNGUUcqIqliZvPcPaeq/TQuo57JzQ0jboqKwvKqVXDzzdCwIdx2W5jJqmXLaGMUkego\n6dczt9++M+HH239/uOee2o9HRNKLqmzWM6tWJV6/ZpcxUyKSiZT065GFCyt+kEqzV4kIKOnXC+7w\n+OPQpw80bw6NG5fd3qxZeABLRERJv47bvBl++tMwqUm/frBkCfzlL9C5cxiK2bkzTJoEw4ZFHamI\npAPdyK3DPvwQLrkEli+Hu++G0aPDzFXDhinJi0hiaunXQe6hXMIxx8B338E774RRO5qqUESqoqRf\nx2zaFGauuv76MCH5hx/CiSfCguQBAAAM30lEQVRGHZWI1BVK+nVIfj707g3/+79hMvLXXgPVpxOR\n6lDSrwPc4aGH4Ljjwny1M2aEJ2v30r+eiFST0kaa27gRLrwQbroJBg6EDz4IyV9EZHco6aexOXOg\nVy+YNg0eeABeeSUUTBMR2V1K+mnIPST5fv3C8syZcMstKoEsIntO4/TTzIYNMHw4vPoqnH9+eNCq\nVauooxKR+kIt/TQya1bozvnb38KN2xdfVMIXkdRS0k8DO3bAH/4Qxts3bBiS/w03qDtHRFJP3TsR\nKyyEn/8cpk8PJRUmTdIkJyJSc5T0IzRjBgwdGvrxH3kkTGGo1r2I1CR170Rg+/ZQIK1//1AKefZs\nuPZaJXwRqXlq6deydetCKeS33oLLLoM//xn22SfqqEQkUyjp16K33w6J/ptvwqQnV16p1r2I1K6k\nunfMbKCZLTGzZWY2upL9LjIzN7Oc2HKWmW0xsw9jP39OVeB1yfbtcOedcNpp0Lo1zJ0LV12lhC8i\nta/Klr6ZNQAmAqcDBUCemU1198Xl9tsHuBGYU+4Uy929Z4rirXPWrAkTmrz7bnjo6o9/DP34IiJR\nSKal3wdY5u4r3P0HYAowOMF+dwH3At+nML46JTcXsrJC9cusrDCTVc+eoWX/1FPwxBNK+CISrWSS\nfnvgs7jlgti6UmbWC+jo7q8mOL6LmX1gZv80sxMSXcDMRppZvpnlFxYWJht7WsnNhZEjYdWqUDtn\n1arwwFXjxqEO/s9/HnWEIiLJJf1EPc9eutFsL+BB4NYE+60FOrl7L+AW4Bkz23eXk7lPcvccd89p\nV0dnBbn9digq2nW9GRx+eO3HIyKSSDJJvwDoGLfcAVgTt7wP0B1418xWAscAU80sx923uvsGAHef\nBywHDklF4Olm9erE6wsKajcOEZHKJJP084CuZtbFzBoBQ4CpJRvdfZO7t3X3LHfPAmYDg9w938za\nxW4EY2Y/BroCK1L+LtJARX+gdOpUu3GIiFSmytE77l5sZqOAN4EGwGR3X2RmY4F8d59ayeEnAmPN\nrBjYDlzr7l+lIvB0MnFiqKFjFvrzSzRrBuPGRReXiEh55vFZKg3k5OR4fn5+1GEkZfv2MLnJQw/B\noEHh5667QldPp04h4Q8bFnWUIpIJzGyeu+dUtZ+eyN1NmzeHYmmvvRYS/733QoMG4aErEZF0paS/\nGz77DM45BxYtCrVzrrkm6ohERJKjpF9N+flw7rlheObrr8OAAVFHJCKSPJVWroaXXgqzWzVuHGa3\nUsIXkbpGST8J7nDffXDhhZCdDXPmQLduUUclIlJ9SvpV2LYtlFf49a/h4otDeeQf/SjqqEREdo+S\nfiW+/hrOPDPUvr/jDnj2WWjaNOqoRER2n27kVmDFCjj7bFi+HJ58Ei6/POqIRET2nJJ+ArNmweDB\nsGMH/P3vcNJJUUckIpIa6t4p59ln4ZRToFWrMGG5Er6I1CdK+jHuMHZsmMO2b1/417+ga9eooxIR\nSS117wBbt8KIEfD006HvftIkaNQo6qhERFIv41v669eHCcuffjoUSHviCSV8Eam/Mrql/8knoYZO\nQQE89xxccknUEYmI1KyMTfpvvx2esG3UCN59F445JuqIRERqXkZ270yeDGecAQcdFEoqKOGLSKbI\nqKS/YweMHh1q3vfvH8bjZ2VFHZWISO3JmO6doiL4+c/hxRdD/fuHH4aGDaOOSkSkdmVE0v/iizCV\nYX4+jB8Pv/xlmM9WRCTT1Pukv3BhGKGzfj28/HJI/iIimape9+lPnw7HHw/FxfDee0r4IiL1NulP\nnBiqZP7kJzB3LvTuHXVEIiLRSyrpm9lAM1tiZsvMbHQl+11kZm5mOXHrbosdt8TMzkhF0JXZvj30\n2Y8aFZL+jBnQvn1NX1VEpG6osk/fzBoAE4HTgQIgz8ymuvvicvvtA9wIzIlbdwQwBOgGHAS8ZWaH\nuPv21L2FnTZvhqFD4bXX4OabwxSHDRrUxJVEROqmZFr6fYBl7r7C3X8ApgCDE+x3F3Av8H3cusHA\nFHff6u6fAsti50u5NWvghBNCP/6f/hRG6Sjhi4iUlUzSbw98FrdcEFtXysx6AR3d/dXqHhs7fqSZ\n5ZtZfmFhYVKBl9e8efh57TW47rrdOoWISL2XzJDNRCPavXSj2V7Ag8Dw6h5busJ9EjAJICcnZ5ft\nyWjZEmbO1Ph7EZHKJJP0C4COccsdgDVxy/sA3YF3LWTcA4CpZjYoiWNTSglfRKRyyXTv5AFdzayL\nmTUi3JidWrLR3Te5e1t3z3L3LGA2MMjd82P7DTGzxmbWBegKzE35uxARkaRU2dJ392IzGwW8CTQA\nJrv7IjMbC+S7+9RKjl1kZs8Di4Fi4Bc1NXJHRESqZu671YVeY3Jycjw/Pz/qMERE6hQzm+fuOVXt\nV2+fyBURkV0p6YuIZBAlfRGRDKKkLyKSQZT0RUQyiJK+iEgGUdIXEckgSvoiIhlESV9EJIMo6YuI\nZBAlfRGRDKKkLyKSQZT0RUQyiJK+iEgGUdIXEckgSvoiIhlESV9EJIMo6YuIZBAlfRGRDKKkLyKS\nQZT0RUQySFJJ38wGmtkSM1tmZqMTbL/WzBaa2YdmNtPMjoitzzKzLbH1H5rZn1P9BkREJHl7V7WD\nmTUAJgKnAwVAnplNdffFcbs94+5/ju0/CBgPDIxtW+7uPVMbtoiI7I5kWvp9gGXuvsLdfwCmAIPj\nd3D3b+IWmwOeuhBFRCRVkkn67YHP4pYLYuvKMLNfmNly4F7gxrhNXczsAzP7p5mdkOgCZjbSzPLN\nLL+wsLAa4YuISHUkk/QtwbpdWvLuPtHdDwZ+A9wRW70W6OTuvYBbgGfMbN8Ex05y9xx3z2nXrl3y\n0YuISLUkk/QLgI5xyx2ANZXsPwU4D8Ddt7r7htjrecBy4JDdC1VERPZUMkk/D+hqZl3MrBEwBJga\nv4OZdY1bPBtYGlvfLnYjGDP7MdAVWJGKwEVEpPqqHL3j7sVmNgp4E2gATHb3RWY2Fsh396nAKDM7\nDdgGbAQujx1+IjDWzIqB7cC17v5VTbwRERGpmrmn10CbnJwcz8/PjzoMEZE6xczmuXtOVfvpiVwR\nkQyipC8ikkGU9EVEMoiSvohIBlHSFxHJIEr6IiIZRElfRCSDKOmLiGQQJX0RkQxSb5J+bi5kZcFe\ne4XfublRRyQikn6qrL1TF+TmwsiRUFQUlletCssAw4ZFF5eISLqpFy3922/fmfBLFBWF9SIislO9\nSPqrV1dvvYhIpqoXSb9Tp+qtFxHJVPUi6Y8bB82alV3XrFlYLyIiO9WLpD9sGEyaBJ07g1n4PWmS\nbuKKiJRXL0bvQEjwSvIiIpWrFy19ERFJjpK+iEgGUdIXEckgSvoiIhlESV9EJIOYu0cdQxlmVgis\nijqOPdQWWB91EGlEn0dZ+jx20mdR1p58Hp3dvV1VO6Vd0q8PzCzf3XOijiNd6PMoS5/HTvosyqqN\nz0PdOyIiGURJX0Qkgyjp14xJUQeQZvR5lKXPYyd9FmXV+OehPn0RkQyilr6ISAZR0hcRySBK+ilk\nZh3N7B0z+9jMFpnZTVHHFDUza2BmH5jZq1HHEjUz28/MXjCzT2L/jRwbdUxRMrObY/+ffGRmz5pZ\nk6hjqk1mNtnMvjSzj+LWtTazv5vZ0tjvVqm+rpJ+ahUDt7r74cAxwC/M7IiIY4raTcDHUQeRJv4b\nmO7uhwHZZPDnYmbtgRuBHHfvDjQAhkQbVa17EhhYbt1o4B/u3hX4R2w5pZT0U8jd17r7+7HXmwn/\nU7ePNqromFkH4Gzg8ahjiZqZ7QucCPwFwN1/cPevo40qcnsDTc1sb6AZsCbieGqVu88Aviq3ejDw\nVOz1U8B5qb6ukn4NMbMsoBcwJ9pIIjUB+DWwI+pA0sCPgULgiVh31+Nm1jzqoKLi7p8D9wOrgbXA\nJnf/W7RRpYUfuftaCI1IYP9UX0BJvwaYWQvgReCX7v5N1PFEwczOAb5093lRx5Im9gZ6A4+4ey/g\nO2rgT/e6ItZXPRjoAhwENDezn0YbVWZQ0k8xM2tISPi57v6/UccToeOBQWa2EpgCnGJmT0cbUqQK\ngAJ3L/nL7wXCl0CmOg341N0L3X0b8L/AcRHHlA7WmdmBALHfX6b6Akr6KWRmRuiz/djdx0cdT5Tc\n/TZ37+DuWYQbdG+7e8a25Nz9C+AzMzs0tupUYHGEIUVtNXCMmTWL/X9zKhl8YzvOVODy2OvLgVdS\nfYF6MzF6mjge+Bmw0Mw+jK37rbu/HmFMkj5uAHLNrBGwArgi4ngi4+5zzOwF4H3CqLcPyLCSDGb2\nLHAy0NbMCoA7gd8Dz5vZVYQvxotTfl2VYRARyRzq3hERySBK+iIiGURJX0Qkgyjpi4hkECV9EZEM\noqQvIpJBlPRFRDLI/wd/f31GKm5K6gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEICAYAAABBBrPDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VdW5//HPQ4LMIkRUSpSgUGUw\nTBHpDxUUVHAApKggVLQot1bbWlsV9Dq29Kq1QB0rtVqqUaRYlGsR60BFvRYIMmhAFBkjCIiCIIgC\nz++PtYOHmJycjCfD9/16nVfOXnvttdc+gTxnr7XXWubuiIiIFKVOsisgIiJVmwKFiIjEpUAhIiJx\nKVCIiEhcChQiIhKXAoWIiMSlQCEVzsxSzGynmR1TnnmTyczamlm5P1tuZv3MbE3M9gozOzWRvKU4\n16NmdlNpj49T7m/N7K/lXa4kT2qyKyBVj5ntjNlsCOwB9kXb/+Xu2SUpz933AY3LO29t4O7Hl0c5\nZnYFMNLd+8SUfUV5lC01nwKFfIe7H/hDHX1jvcLdXykqv5mluvveyqibiFQ+NT1JiUVNC8+Y2dNm\ntgMYaWY/MLP/mNk2M9toZveZWd0of6qZuZllRNtPRvtfNLMdZva2mbUpad5o/wAz+8DMtpvZ/Wb2\nlpldVkS9E6njf5nZSjP73Mzuizk2xcwmmtlWM/sI6B/n8/lvM5taIO1BM5sQvb/CzJZH1/NR9G2/\nqLLyzKxP9L6hmT0R1S0X6F7IeVdF5eaa2cAo/UTgAeDUqFnv05jP9vaY438SXftWM3vOzFom8tkU\nx8wGR/XZZmavmdnxMftuMrMNZvaFmb0fc609zeydKH2Tmf0+0fNJBXB3vfQq8gWsAfoVSPst8DVw\nPuHLRgPgJOBkwl3qscAHwDVR/lTAgYxo+0ngUyALqAs8AzxZirxHADuAQdG+64BvgMuKuJZE6vg8\n0BTIAD7Lv3bgGiAXSAfSgLnhv0+h5zkW2Ak0iil7M5AVbZ8f5THgDGA3kBnt6wesiSkrD+gTvb8X\n+DfQDGgNLCuQ9yKgZfQ7uSSqw5HRviuAfxeo55PA7dH7s6I6dgHqAw8BryXy2RRy/b8F/hq9bx/V\n44zod3RT9LnXBToCa4GjorxtgGOj9wuA4dH7JsDJyf6/UJtfuqOQ0nrT3f/X3fe7+253X+Du89x9\nr7uvAiYDveMcP93dc9z9GyCb8AeqpHnPAxa7+/PRvomEoFKoBOv4P+6+3d3XEP4o55/rImCiu+e5\n+1bgrjjnWQW8RwhgAGcC29w9J9r/v+6+yoPXgFeBQjusC7gI+K27f+7uawl3CbHnnebuG6PfyVOE\nIJ+VQLkAI4BH3X2xu38FjAV6m1l6TJ6iPpt4hgEz3f216Hd0F3AoIWDvJQSljlHz5eros4MQ8NuZ\nWZq773D3eQleh1QABQoprfWxG2Z2gpn908w+MbMvgDuBw+Mc/0nM+13E78AuKu/3Yuvh7k74Bl6o\nBOuY0LkI34TjeQoYHr2/hBDg8utxnpnNM7PPzGwb4dt8vM8qX8t4dTCzy8xsSdTEsw04IcFyIVzf\ngfLc/Qvgc6BVTJ6S/M6KKnc/4XfUyt1XAL8i/B42R02ZR0VZLwc6ACvMbL6ZnZPgdUgFUKCQ0ir4\naOgjhG/Rbd39UOBWQtNKRdpIaAoCwMyMg/+wFVSWOm4Ejo7ZLu7x3WeAftE38kGEwIGZNQCmA/9D\naBY6DPhXgvX4pKg6mNmxwMPAVUBaVO77MeUW9yjvBkJzVn55TQhNXB8nUK+SlFuH8Dv7GMDdn3T3\nXoRmpxTC54K7r3D3YYTmxT8Az5pZ/TLWRUpJgULKSxNgO/ClmbUH/qsSzvkC0M3MzjezVOAXQIsK\nquM04Foza2VmacCN8TK7+ybgTeBxYIW7fxjtqgccAmwB9pnZeUDfEtThJjM7zMI4k2ti9jUmBIMt\nhJh5BeGOIt8mID2/874QTwOjzSzTzOoR/mC/4e5F3qGVoM4DzaxPdO7rCf1K88ysvZmdHp1vd/Ta\nR7iAH5nZ4dEdyPbo2vaXsS5SSgoUUl5+BYwi/BF4hPCNukJFf4wvBiYAW4HjgEWEcR/lXceHCX0J\n7xI6WqcncMxThM7pp2LqvA34JTCD0CE8lBDwEnEb4c5mDfAi8LeYcpcC9wHzozwnALHt+i8DHwKb\nzCy2CSn/+NmEJqAZ0fHHEPotysTdcwmf+cOEINYfGBj1V9QD7iH0K31CuIP57+jQc4DlFp6quxe4\n2N2/Lmt9pHQsNOuKVH9mlkJo6hjq7m8kuz4iNYXuKKRaM7P+ZtY0ar64hfAkzfwkV0ukRlGgkOru\nFGAVofmiPzDY3YtqehKRUlDTk4iIxKU7ChERiatGTAp4+OGHe0ZGRrKrISJSrSxcuPBTd4/3SDlQ\nQwJFRkYGOTk5ya6GiEi1YmbFzTAAqOlJRESKoUAhIiJxKVCIiEhcNaKPQkQq1zfffENeXh5fffVV\nsqsiCahfvz7p6enUrVvUVF/xKVCISInl5eXRpEkTMjIyCJP2SlXl7mzdupW8vDzatGlT/AGFqLVN\nT9nZkJEBdeqEn9nZxR0hIvm++uor0tLSFCSqATMjLS2tTHd/tfKOIjsbxoyBXbvC9tq1YRtgRJnn\nyxSpHRQkqo+y/q5q5R3FzTd/GyTy7doV0kVE5GC1MlCsW1eydBGpWrZu3UqXLl3o0qULRx11FK1a\ntTqw/fXXhS9bcfbZZ7Njx4645d58883MmTOnXOqYnp7Otm3byqWsZKuVTU/HHBOamwpLF5Hyl50d\n7tjXrQv/z8aPL1szb1paGosXLwbg9ttvp3Hjxvz617+Oe8xLL71UbLnjx48vfaVqsFp5RzF+PDRs\neHBaw4YhXUTKV36f4Nq14P5tn2BFPUBy/vnn0717dzp27Mijjz56ID3/G/7KlSvp1KkTo0ePpmPH\njgwYMOBAR+/IkSN57rnnDuS//fbb6dq1K5mZmXzwwQcAbN68mb59+9KtWzd++tOf0qpVq2LvHO65\n5x46depEp06duP/++wHYsWMHAwYMoHPnznTq1Inp08Oiiddffz0dOnQgMzOTG2+Mu+JupamVgWLE\nCJg8GVq3BrPwc/JkdWSLVITK7hOcMmUKCxcuZMGCBUyYMIHPP//8O3lWrFjBtddeS25uLg0aNDgQ\nHAo68sgjWbRoEVdccQUTJkwA4NZbb6V///688847nHPOOWzYsCFufebPn092djbz58/n7bff5qGH\nHmLp0qXMmjWLjIwMlixZwnvvvceZZ57Jpk2bmDVrFrm5uSxdupRx48aV/QMpB7UyUEAICmvWwP79\n4aeChEjFqOw+wYkTJ9K5c2d+8IMfkJeXx0cfffSdPG3btuXEE08EoHv37qxZs6bQsoYMGfKdPG++\n+SbDhg0D4LzzzqNJkyZx6/PGG2/wwx/+kIYNG9KkSRMGDx7Mm2++SWZmJrNnz2bs2LG89dZbNG3a\nlObNm1OnTh2uvPJKZsyYQaNGjUr5KZSvWhsoRKRyFNX3VxF9gq+88gpz587lP//5D0uWLCEzM7PQ\n8QP16tU78D4lJYW9e/cWWl5+vtg8JV3sraj87du3Jycnh44dO3L99dfzu9/9jrp165KTk8PgwYN5\n9tlnOffcc0t0roqiQCEiFaoy+wS3b99O8+bNadCgAbm5uSxYsKDcz3HKKacwbdo0AGbNmlXsk1Sn\nnXYaM2bMYPfu3ezcuZPnn3+eU089lY8//pjGjRvzox/9iOuuu4533nmHHTt28MUXX3DeeecxceJE\nFi1aVO71L41a+dSTiFSe/Gbd8nzqqSjnnnsukydPpnPnzpxwwgmcfPLJ5X6OO+64g0suuYTs7GzO\nOOMMjjzyyLhNRD169GD48OGcdNJJAFx11VWceOKJzJo1i7Fjx1KnTh0OOeQQ/vSnP7F9+3aGDBnC\nnj172L9//4F+kWSrEWtmZ2VluRYuEqk8y5cvp3379smuRlJ89dVXpKamkpqayptvvsm1115bLRZO\nK+x3ZmYL3T2ruGN1RyEiUgJr1qxh+PDh7Nu3j3r16vHII48ku0oVLqE+CjPrb2YrzGylmY0tZH89\nM3sm2j/PzDJi9o2L0leY2dlR2vFmtjjm9YWZXRvta25mL5vZh9HPZuVzqSIiZXfCCSewaNEili5d\nyoIFC+jevXuyq1Thig0UZpYCPAgMADoAw82sQ4Fso4HP3b0tMBG4Ozq2AzAM6Aj0Bx4ysxR3X+Hu\nXdy9C9Ad2AXMiMoaC7zq7u2AV6NtERFJkkTuKHoAK919lbt/DUwFBhXIMwiYEr2fDvS1MF3hIGCq\nu+9x99XAyqi8WH2Bj9x9bSFlTQEGl+SCRESkfCUSKFoB62O286K0QvO4+15gO5CW4LHDgKdjto90\n941RWRuBIwqrlJmNMbMcM8vZsmVLApchIiKlkUigKGwi84KPShWVJ+6xZnYIMBD4ewL1OLgQ98nu\nnuXuWS1atCjp4SIikqBEAkUecHTMdjpQcHKTA3nMLBVoCnyWwLEDgHfcfVNM2iYzaxmV1RLYnEAd\nRaQW6dOnz3dmg500aRI//elP4x7XuHFjADZs2MDQoUOLLLu4x10nTZrErpgJrM4555xymVL89ttv\n59577y1zOeUtkUCxAGhnZm2iO4BhwMwCeWYCo6L3Q4HXPAzQmAkMi56KagO0A+bHHDecg5udCpY1\nCng+0YsRkdph+PDhTJ069aC0qVOnMnz48ISO/973vndgttbSKBgoZs2axWGHHVbq8qq6YgNF1Odw\nDfASsByY5u65ZnanmQ2Msv0FSDOzlcB1RE8quXsuMA1YBswGrnb3fQBm1hA4E/hHgVPeBZxpZh9G\n++8q2yWKSE0zdOhQXnjhBfbs2QOEsQ0bNmzglFNOYefOnQemAT/xxBN5/vnvftdcs2YNnTp1AmD3\n7t0MGzaMzMxMLr74Ynbv3n0g31VXXUVWVhYdO3bktttuA+C+++5jw4YNnH766Zx++ukAZGRk8Omn\nnwIwYcKEA1OKT5o06cD52rdvz5VXXknHjh0566yzDjpPYRYvXkzPnj3JzMzkggsuODAL7n333Xdg\nGvL8yQlff/31Aws3de3atdhpRUoqoQF37j4LmFUg7daY918BFxZx7HjgO7O6uPsuQod3wfSthCeh\nRKQauPZaiNYQKjddukD0N7ZQaWlp9OjRg9mzZzNo0CCmTp3KxRdfjJlRv359ZsyYwaGHHsqnn35K\nz549GThwYJHrRj/88MM0bNiQpUuXsnTpUrp163Zg3/jx42nevDn79u2jb9++LF26lJ///OdMmDCB\nOXPmcPjhhx9U1sKFC3n88ceZN28e7s7JJ59M7969adasGR9++CFPP/00f/7zn7nooot49tlnGTly\nZJHXeOmll3L//ffTu3dvbr31Vu644w4mTZrEXXfdxerVq6lXr96B5q57772XBx98kF69erFz507q\n169fgk+7eJoUUESqpdjmp9hmJ3fnpptuIjMzk379+vHxxx+zadOmIsuZO3fugT/YmZmZZGZmHtg3\nbdo0unXrRteuXcnNzWXZsmVx6/Tmm29ywQUX0KhRIxo3bsyQIUN44403AGjTpg1dunQB4k9tDmFy\nw23bttG7d28ARo0axdy5cw/UccSIETz55JOkpobv+r169eK6667jvvvuY9u2bQfSy4um8BCRMon3\nzb8iDR48+MCsq7t37z5wJ5Cdnc2WLVtYuHAhdevWJSMjo9CpxmMVdrexevVq7r33XhYsWECzZs24\n7LLLii0n3tx5Bac2L67pqSj//Oc/mTt3LjNnzuQ3v/kNubm5jB07lnPPPZdZs2bRs2dPXnnlFU44\n4YRSlV8Y3VGISLXUuHFj+vTpw49//OODOrG3b9/OEUccQd26dZkzZw5r166NU0qYBjw7Wpf1vffe\nY+nSpQB88cUXNGrUiKZNm7Jp0yZefPHFA8c0adKk0H6A0047jeeee45du3bx5ZdfMmPGDE499dQS\nX1vTpk1p1qzZgbuRJ554gt69e7N//37Wr1/P6aefzj333MO2bdvYuXMnH330ESeeeCI33ngjWVlZ\nvP/++yU+Zzy6oxCRamv48OEMGTLkoCegRowYwfnnn09WVhZdunQp9pv1VVddxeWXX05mZiZdunSh\nR48weUTnzp3p2rUrHTt25Nhjj6VXr14HjhkzZgwDBgygZcuWzJkz50B6t27duOyyyw6UccUVV9C1\na9e4zUxFmTJlCj/5yU/YtWsXxx57LI8//jj79u1j5MiRbN++HXfnl7/8JYcddhi33HILc+bMISUl\nhQ4dOjBgwIASny8eTTMuIiVWm6cZr67KMs24mp5ERCQuBQoREYlLgUJESqUmNFvXFmX9XSlQiEiJ\n1a9fn61btypYVAPuztatW8s0CE9PPYlIiaWnp5OXl4em+K8e6tevT3p6eqmPV6AQkRKrW7cubdq0\nSXY1pJKo6UlEROJSoBARkbgUKEREJC4FChERiUuBQkRE4lKgEBGRuBQoREQkLgUKERGJS4FCRETi\nUqAQEZG4EgoUZtbfzFaY2UozG1vI/npm9ky0f56ZZcTsGxelrzCzs2PSDzOz6Wb2vpktN7MfROm3\nm9nHZrY4ep1T9ssUEZHSKjZQmFkK8CAwAOgADDezDgWyjQY+d/e2wETg7ujYDsAwoCPQH3goKg/g\nj8Bsdz8B6Awsjylvort3iV6zSn11xdiwAWbPrqjSRURqhkTuKHoAK919lbt/DUwFBhXIMwiYEr2f\nDvQ1M4vSp7r7HndfDawEepjZocBpwF8A3P1rd99W9sspmRtugIEDYVaFhSIRkeovkUDRClgfs50X\npRWax933AtuBtDjHHgtsAR43s0Vm9qiZNYrJd42ZLTWzx8ysWWGVMrMxZpZjZjmlner4gQfgxBNh\nyBB49dVSFSEiUuMlEiiskLSCq5UUlaeo9FSgG/Cwu3cFvgTy+z4eBo4DugAbgT8UVil3n+zuWe6e\n1aJFi2IvojCHHQb/+he0axfuLN56q1TFiIjUaIkEijzg6JjtdGBDUXnMLBVoCnwW59g8IM/d50Xp\n0wmBA3ff5O773H0/8GdC01eFSUuDl1+G9HQYMAAWLKjIs4mIVD+JBIoFQDsza2NmhxA6p2cWyDMT\nGBW9Hwq85mGNxJnAsOipqDZAO2C+u38CrDez46Nj+gLLAMysZUy5FwDvleK6SuSoo0LT0+GHw9ln\nw9KlFX1GEZHqo9gV7tx9r5ldA7wEpACPuXuumd0J5Lj7TEKn9BNmtpJwJzEsOjbXzKYRgsBe4Gp3\n3xcV/TMgOwo+q4DLo/R7zKwLoYlqDfBf5XOp8aWnh2Bx2mnQrx+8/jq0b18ZZxYRqdqsJiyOnpWV\n5Tk5OeVS1gcfhGBRpw7MnQtt25ZLsSIiVY6ZLXT3rOLyaWR2Ad//PrzyCnz9NfTtC+vWJbtGIiLJ\npUBRiE6dwtNQ27fDGWeEgXkiIrWVAkURunULo7Y3bQp9FqUcqiEiUu0pUMTRsye88AKsWQNnngmf\nfZbsGomIVD4FimL07g3PPQfLl0P//vDFF8mukYhI5VKgSMBZZ8H06bBoEZx7Lnz5ZbJrJCJSeRQo\nEnT++ZCdDf/3fzBoEHz1VbJrJCJSORQoSuCii+Dxx+G112Do0PAIrYhITadAUUKXXgp/+hP8859w\nySWwd2+yayQiUrEUKEphzBiYOBGefRZGjYJ9+4o/RkSkuip2ricp3LXXwu7dcNNN0LAhPPJImPZD\nRKSmUaAog3HjYNcu+O1voUED+OMfwQpbgUNEpBpToCijO+8MwWLChBAs7rpLwUJEahYFijIyg3vv\nDc1Q99wTmqFuuy3ZtRIRKT8KFOXALKy/vXs33H57uLO44YZk10pEpHwoUJSTOnXg0UdDsLjxxnBn\ncc01ya6ViEjZKVCUo5QUeOKJMGr7Zz8LdxajRye7ViIiZaMHOstZ3brwzDNhAsErr4Snnkp2jURE\nykaBogLUqwf/+Af06RNGcj/7bLJrJCJSegoUFaRBA5g5E3r0gOHDw5QfIiLVkQJFBWrcGF58ETIz\n4Yc/hFdfTXaNRERKLqFAYWb9zWyFma00s7GF7K9nZs9E++eZWUbMvnFR+gozOzsm/TAzm25m75vZ\ncjP7QZTe3MxeNrMPo5/Nyn6ZydO0Kbz0ErRrBwMHwptvJrtGIiIlU2ygMLMU4EFgANABGG5mHQpk\nGw187u5tgYnA3dGxHYBhQEegP/BQVB7AH4HZ7n4C0BlYHqWPBV5193bAq9F2tZaWBq+8AkcfDeec\nAwsWJLtGIiKJS+SOogew0t1XufvXwFRgUIE8g4Ap0fvpQF8zsyh9qrvvcffVwEqgh5kdCpwG/AXA\n3b92922FlDUFGFy6S6tajjwyND21aAFnnw1LliS7RiIiiUkkULQC1sds50VpheZx973AdiAtzrHH\nAluAx81skZk9amaNojxHuvvGqKyNwBEluqIqrFWrECwaNYJ+/WDZsrBqXkZGGLCXkRG2RUSqkkQC\nRWFT3HmCeYpKTwW6AQ+7e1fgS0rYxGRmY8wsx8xytmzZUpJDkyojI6yQl5oKvXrBFVfA2rXgHn6O\nGaNgISJVSyKBIg84OmY7HdhQVB4zSwWaAp/FOTYPyHP3eVH6dELgANhkZi2jsloCmwurlLtPdvcs\nd89q0aJFApdRdbRrF/osvvjiu2tv79oFN9+cnHqJiBQmkUCxAGhnZm3M7BBC5/TMAnlmAqOi90OB\n19zdo/Rh0VNRbYB2wHx3/wRYb2bHR8f0BZYVUtYo4PlSXFeV17Ej7N9f+L516yq3LiIi8RQ715O7\n7zWza4CXgBTgMXfPNbM7gRx3n0nolH7CzFYS7iSGRcfmmtk0QhDYC1zt7vkLh/4MyI6Czyrg8ij9\nLmCamY0G1gEXltO1VjmtW4fmpoKOOaby6yIiUhQLX/yrt6ysLM/JyUl2NUosOzv0SezadXD6mWfC\n5MmhP0NEpKKY2UJ3zyoun0ZmJ9GIESEgtG4d1rT43vfgtNPg3/+Gtm3hRz+Cd99Ndi1FpLZToEiy\nESNgzZrQX/Hxx/D667BqFfziFzBjRpj+4/zzNaJbRJJHgaIKSk+HP/whdGrfeSe8/Taceiqccgq8\n8EJ4lFZEpLIoUFRhzZvDLbeEDu/77oP168PdRWZm6N/YuzfZNRSR2kCBohpo1CismLdyJfztb+GO\nYuTI0I/xwAPf7QwXESlPChTVSN26oYN76dKw1kWrViGAtG4Nv/0tfP55smsoIjWRAkU1VKdOaIJ6\n6y144w04+eTQRHXMMfDrX4dOcRGR8qJAUc3ld3AvWQKDBsGkSdCmTZhDasWKZNdORGoCBYoaIjMT\nnnwSPvzw24kF27eHoUO1/oWIlI0CRQ3Tpk3o4F67Fm66KUxr3qNHmNb8lVf0aK2IlJwCRQ11xBGh\ng3vtWvj978PaF2eeCSedBNOnw759xZchIgIKFDXeoYeGDu7Vq+HPfw5Tm194YWiWevRR2LMn2TUU\nkapOgaKWqFcvdHAvXx7uKA49FK68MjRV/f73IYCIiBRGgaKWSUmBH/4wdHC/8kpYF+OGG8KjtTff\nDFu3JruGIlLVKFDUUmbQty+8/HIIGmeeCf/zP9C5cxifISKST4FCyMqCv/89BIx69aB3b7j3Xj0h\nJSKBAoUc0L07vPNOGLh3/fUweLCmBRERBQopoGnT0Nk9aRLMmgXdumnAnkhtp0Ah32EWFk56442w\noFKvXmEQn5qiRGonBQopUs+eoSnqrLPCLLUXX6zHaEVqIwUKiSstLUxpfvfd8I9/hH6MJUuSXSsR\nqUwKFFKsOnXCWIs5c8IiSSefHEZ5qylKpHZIKFCYWX8zW2FmK81sbCH765nZM9H+eWaWEbNvXJS+\nwszOjklfY2bvmtliM8uJSb/dzD6O0heb2Tllu0QpL6eeCosWhZ9jxsCll8LOncmulYhUtGIDhZml\nAA8CA4AOwHAz61Ag22jgc3dvC0wE7o6O7QAMAzoC/YGHovLyne7uXdw9q0B5E6P0Lu4+qzQXJiWT\nnQ0ZGeHuISMjbBfmiCNg9my4446Qp0ePMOGgiNRcidxR9ABWuvsqd/8amAoMKpBnEDAlej8d6Gtm\nFqVPdfc97r4aWBmVJ1VIdna4Q1i7NjQnrV377ZoWhUlJgVtvDaO6t24NM9I+8UTl1llEKk8igaIV\nsD5mOy9KKzSPu+8FtgNpxRzrwL/MbKGZjSlQ3jVmttTMHjOzZoVVyszGmFmOmeVs2bIlgcuQotx8\nc+h7iLVrV0iPp29fWLw4BIpLLw2TDO7eXXH1FJHkSCRQWCFpBbsxi8oT79he7t6N0KR1tZmdFqU/\nDBwHdAE2An8orFLuPtnds9w9q0WLFsVcgsSzbl3J0mO1bBkmF7zppjBtec+e8MEH5Vs/EUmuRAJF\nHnB0zHY6sKGoPGaWCjQFPot3rLvn/9wMzCBqknL3Te6+z933A39GTVUV7phjSpZeUGoqjB8fRnLn\n5YW5o6ZNK7/6iUhyJRIoFgDtzKyNmR1C6JyeWSDPTGBU9H4o8Jq7e5Q+LHoqqg3QDphvZo3MrAmA\nmTUCzgLei7ZbxpR7QX66VJzx46Fhw4PTGjYM6SUxYEBoiurUKQzOu+YaLYwkUhMUGyiiPodrgJeA\n5cA0d881szvNbGCU7S9AmpmtBK4DxkbH5gLTgGXAbOBqd98HHAm8aWZLgPnAP919dlTWPdFjs0uB\n04FfltO1ShFGjIDJk6F16zB9R+vWYXvEiJKXdfTR8Prr8KtfwYMPhuk/Vq8u/zqLSOUxrwGjprKy\nsjwnJ6f4jFKpnnsOLrssvJ8yJcxKKyJVh5ktLGR4wndoZLZUmMGDw1xRbduG97/6FXzzTbJrJSIl\npUAhFerYY8OKeddcAxMmhEWR1q8v/jgRqToUKKTC1asH998PzzwD770HXbvCiy8mu1YikigFCqk0\nF10EOTnQqhWcc04Y0Ld3b7JrJSLFUaCQSvX978N//gNXXAG/+x306wcbNya7ViISjwKFVLoGDcI0\n5VOmhGVWu3SB115Ldq1EpCgKFJI0l14K8+eHxZH69YM774R9+5JdKxEpSIFCkqpjxxAsRoyA224L\no7s3b052rUQklgKFJF3jxvDtOnjWAAASVElEQVS3v4XmqLlzw1NRU6fq7kKkqlCgkCrBLHRwz5sH\nzZvD8OGQmRkeqVXAEEkuBQqpUjp3DhMLTp0aFlEaNuzbgLF/f7JrJ1I7KVBIlZKdDccdF+4ovvwy\njOiODRjTpilgiFQ2BQqpMgouybpuHTz2GIwbB08/HQLExReHgPH3vytgiFQWBQqpMopakvWWW8Id\nxbvvwlNPhT6Liy5SwBCpLAoUUmUUtyRrSkpoknrvvYMDRufOMH26AoZIRVGgkCoj0SVZYwNGdnaY\nuvzCC8MI72efVcAQKW8KFFJllHRJ1pQUuOQSyM0NAePrr2HoUAUMkfKmQCFVRmmXZI0NGE8+Gdbp\nHjo0DNz7xz8UMETKSoFCqpQRI2DNmvDHfc2akq3bnZIS8i9bFgLGV1/BD3+ogCFSVgoUUuPkB4zc\nXHjiCdi9OwSMbt1gxgwFDJGSUqCQGis1FUaODHcYf/tbeNR2yBDo3h2eey6M1RCR4iUUKMysv5mt\nMLOVZja2kP31zOyZaP88M8uI2TcuSl9hZmfHpK8xs3fNbLGZ5cSkNzezl83sw+hns7JdotR2qanw\nox99GzB27oQLLgh3GAoYIsUrNlCYWQrwIDAA6AAMN7MOBbKNBj5397bARODu6NgOwDCgI9AfeCgq\nL9/p7t7F3bNi0sYCr7p7O+DVaFukzPIDxvLlYdGk2IDx/PMKGCJFSeSOogew0t1XufvXwFRgUIE8\ng4Ap0fvpQF8zsyh9qrvvcffVwMqovHhiy5oCDE6gjiIJS00NiybFBozBg0OT1MyZChgiBSUSKFoB\n62O286K0QvO4+15gO5BWzLEO/MvMFprZmJg8R7r7xqisjcARhVXKzMaYWY6Z5WzZsiWByxA5WGzA\n+Otf4YsvYNAgBQyRghIJFFZIWsH/QkXliXdsL3fvRmjSutrMTkugLt8W4j7Z3bPcPatFixYlOVTk\nIKmpMGoUvP8+PP44bN8eAkbnzvCTn8DEiTBrFnz0kdbGkNopNYE8ecDRMdvpwIYi8uSZWSrQFPgs\n3rHunv9zs5nNIDRJzQU2mVlLd99oZi0BLYwplSI1FS67LDxam50dVtybNg0+//zbPIccAm3bwvHH\nf/v6/vfDz7S0pFVdpEIlEigWAO3MrA3wMaFz+pICeWYCo4C3gaHAa+7uZjYTeMrMJgDfA9oB882s\nEVDH3XdE788C7ixQ1l3Rz+fLcoEiJVW3bggYl10Wmp+2boUVKw5+LV8OL7wQ5pnKl5Z2cADJfx13\nXAgwItVVsYHC3fea2TXAS0AK8Ji755rZnUCOu88E/gI8YWYrCXcSw6Jjc81sGrAM2Atc7e77zOxI\nYEbo7yYVeMrdZ0envAuYZmajgXXAheV4vSLFys4OU56vWxcmJBw/Ptxl9Op1cL69e2H16u8GkRdf\nDE1Y+VJSoE2bb+88Yl9HHRWmKxGpysxrQI9dVlaW5+TkFJ9RpBj5iyfFrovRsGFic07F2r4dPvjg\nu0Hkgw/C1CL5Dj208ADSrt13J0gUKW9mtrDA8ITC8ylQiHwrIyOssFdQ69Zh7qmy2r8f1q8/OHDk\nvy+4HsfRR4eg0a1bCF7HHVf284vEUqAQKYU6dQp/LNas4ueI2rULPvzwu3chS5aEZq6BA+Haa6F3\nbzVXSflINFAk0pktUmscc0zhdxRFLapUnho2DI/kdu58cPrGjfDQQ/CnP4UR5J07h4AxbBjUr1/x\n9RLRpIAiMUq6eFJlaNkSfvOb0DT16KNhLMfll4fmsNtvh02bklc3qR0UKERilHbxpMrQoAGMHg1L\nl8LLL8NJJ8Edd4S7ncsvh8WLk11DqanURyFSjX3wAdx3X3gcd9cu6NMnNEudd154LFcknkT7KHRH\nIVKNff/78MADkJcH99wTphkZPDik//GPsGNHsmsoNYEChUgN0KwZXH89rFoVph056qhwZ5GeDtdd\nFwYGipSWAoVIDZKaChdeCG+9BfPmhSao++8P81MNGQJz52pWXCk5BQqRGqpHjzDSfPVquPFGeP31\nMAaje/ew0t+ePcmuoVQXChQiNVx6Ovzud2FE+COPhClERo0Ko9B/8xvYrPmZpRgKFCK1RMOGYSqQ\n3FyYPRu6dIFbbw2P144eDe++m+waSlWlQCFSRWVnh2/9deqEn9nZ5VOuGZx9dpjldtmyMAbj6ach\nMxP69QvTp1f0dCVSvShQiFRB+bPYrl0bOp/Xrg3b5RUs8rVvDw8/HB6vveuusMrf+eeHyQgfeCCs\nJy6iQCFSBd1888FTnUPYvvnmijlf8+ahw3v16nB30bw5/OxnoX/j178ufP4rqT00MlukCkrmLLb5\n/vMfmDQJpk8PdTnrLOjbN4z+7tIlPIor1ZtmjxWpxpI5i22+nj1h6tTwtNSDD8Jzz4VBfRAWXDr1\n1PC4bZ8+0LWrAkdNpjsKkSqovFbaK28bN4bxGP/+d3itWBHSmzQJgaNPHwWO6kQLF4lUc0Wt3V2V\nxAaO118PneEQAscpp3wbOLp1U+CoihQoRKTSffLJwXcc+YGjceOD7zgUOKoGBQoRSbpPPgnzS+UH\njuXLQ3rjxt+946hbN3n1rK0UKESkytm06eA7DgWO5CrXQGFm/YE/AinAo+5+V4H99YC/Ad2BrcDF\n7r4m2jcOGA3sA37u7i/FHJcC5AAfu/t5Udpfgd7A9ijbZe4ed+0uBQqR6mnTpoPvOJYtC+mNGh0c\nOLp3V+CoCOX2eGz0x/xB4EwgD1hgZjPdfVlMttHA5+7e1syGAXcDF5tZB2AY0BH4HvCKmX3f3fdF\nx/0CWA4cWuC017v79OLqJiLV25FHhmnRL7wwbBcMHOPGhfTYwNGhAxx+eHilpYW1OOpo6HCFSqQ7\nqQew0t1XAZjZVGAQEBsoBgG3R++nAw+YmUXpU919D7DazFZG5b1tZunAucB44LpyuBYRqQCV+fRV\nwcCxeXPhgSNWnTphJHls8Mh/X1Ra06YKLiWRSKBoBayP2c4DTi4qj7vvNbPtQFqU/p8Cx7aK3k8C\nbgCaFHLO8WZ2K/AqMDYKNAcxszHAGIBjKnMUkkgtUnA8R/6cU1A5j+oecQQMHRpeAFu2hDp8+ils\n3Rp+5r/yt1etgvnzw/tvvim83Dp1vhs8igswTZuGkfG1USKBorCPpmDHRlF5Ck03s/OAze6+0Mz6\nFNg/DvgEOASYDNwI3PmdQtwnR/vJysqq/j3yIlVQvDmnkjGmo0WL8EqEe5jUsLBgUnD7ww/h7bfD\n+717Cy8vNfXbO5cWLcLkiVdeGUap13SJBIo84OiY7XRgQxF58swsFWgKfBbn2IHAQDM7B6gPHGpm\nT7r7SHffGOXdY2aPA78u4TWJSDlZt65k6VWJWRj416QJtGmT2DHusGNH8cFlzZowWeJvfgNXXQU/\n/zm0bFmhl5NUiQSKBUA7M2sDfEzonL6kQJ6ZwCjgbWAo8Jq7u5nNBJ4yswmEzux2wHx3f5tw50B0\nR/Frdx8Zbbd0941RH8dg4L0yXqOIlFJVmHOqMpmFO4RDD4Vjj42fd8EC+P3v4Z57YMIEuPTSEDyO\nP75y6lqZiu3Ocfe9wDXAS4QnlKa5e66Z3WlmA6NsfwHSos7q64Cx0bG5wDRCx/ds4OqYJ56Kkm1m\n7wLvAocDvy35ZYlIeRg/PswxFathw5Be2510EkybFua7Gj0annwyrO8xeDD83/8lu3blSwPuRCSu\n6jDnVFWwZUtY7OmBB+Czz6BXL7jhBjjvvKr7hJVGZouIJMGXX8Jjj8Ef/hCa7U44ITRJjRwJ9eol\nu3YHSzRQVNE4JyJSPTVqFFYHXLkyrBbYoAFccUXoUL/7bti2Ldk1LDkFChGRCpCaCsOGwcKF8PLL\n0KkTjB0bmu+uvz6sU15dKFCISLWQnQ0ZGaG9PyMjbFcHZtCvH/zrX/DOO6HPYuLE8FTVZZfBe9Xg\nuU4FChGp8vJHiK9dG8Y65I8Qry7BIl/XrvDUU2GA309+An//O5x4Yggec+cWvk56VaBAISJVXrwR\n4tVRmzZw333hSbI774R588L64z/4ATz7LOwrbhBBJVOgEJEqrzqPEI8nLQ1uuSXcIT30UHjEdujQ\n8KTUI4/A7t3JrmGgQCEiVV5RI8Frygjxhg3DVCAffBAG8R12WGiaysgI41Y++yy59VOgEJEqr7aM\nEE9JCVOsz58Pc+aEBZv++79DQLz22sKnU6kMChQiUuWNGAGTJ0Pr1uEpotatw3ZNHSFuFhZpmjUL\nliyBIUPgwQfhuOPCwL0lSyq5PhqZLSJS9a1fD5MmhQC5cyecfXaYIuT000u/ToZGZouI1CBHHx2m\nBVm3Dn73O1i8GPr2DY/YVjQFChGRaqRZs7Ak7Jo18Je/wMCBxR5SZomsRyEiIlVM/frw4x9Xzrl0\nRyEiInEpUIiIlEB1nXOqLNT0JCKSoPw5p/KnE8mfcwpq7qO6oDsKEZGE1bQ5pxKlQCEikqCaOudU\ncRQoREQSVNPnnCqKAoWISIJqy5xTBSUUKMysv5mtMLOVZja2kP31zOyZaP88M8uI2TcuSl9hZmcX\nOC7FzBaZ2QsxaW2iMj6Myjyk9JcnIlJ+atucU/mKDRRmlgI8CAwAOgDDzaxDgWyjgc/dvS0wEbg7\nOrYDMAzoCPQHHorKy/cLYHmBsu4GJrp7O+DzqGwRkSphxIgwKnr//vCzpgcJSOyOogew0t1XufvX\nwFRgUIE8g4Ap0fvpQF8zsyh9qrvvcffVwMqoPMwsHTgXeDS/kOiYM6IyiMocXJoLExGR8pFIoGgF\nrI/ZzovSCs3j7nuB7UBaMcdOAm4A9sfsTwO2RWUUdS4AzGyMmeWYWc6WLVsSuAwRkZqjMgf+JRIo\nCpvAtuDc5EXlKTTdzM4DNrv7wlKcKyS6T3b3LHfPatGiRWFZRERqpPyBf2vXgvu3A/8qKlgkEijy\ngKNjttOBDUXlMbNUoCnwWZxjewEDzWwNoSnrDDN7EvgUOCwqo6hziYjUapU98C+RQLEAaBc9jXQI\noXN6ZoE8M4FR0fuhwGseVkSaCQyLnopqA7QD5rv7OHdPd/eMqLzX3H1kdMycqAyiMp8vw/WJiNQ4\nlT3wr9hAEfUXXAO8RHhCaZq755rZnWaWPxP6X4A0M1sJXAeMjY7NBaYBy4DZwNXuvq+YU94IXBeV\nlRaVLSIikcoe+KelUEVEqpmCkxNCGPhX0jEdWgpVRKSGquyBf5pmXESkGhoxovIG++mOQkRE4lKg\nEBGRuBQoREQkLgUKERGJS4FCRETiqhHjKMxsC7A22fUoo8MJU5hIoM/jW/osDqbP42Bl+Txau3ux\nk+XViEBRE5hZTiIDX2oLfR7f0mdxMH0eB6uMz0NNTyIiEpcChYiIxKVAUXVMTnYFqhh9Ht/SZ3Ew\nfR4Hq/DPQ30UIiISl+4oREQkLgUKERGJS4EiyczsaDObY2bLzSzXzH6R7Dolm5mlmNkiM3sh2XVJ\nNjM7zMymm9n70b+RHyS7TsliZr+M/o+8Z2ZPm1n9ZNepMpnZY2a22czei0lrbmYvm9mH0c9mFXFu\nBYrk2wv8yt3bAz2Bq82sQ5LrlGy/IKymKPBHYLa7nwB0ppZ+LmbWCvg5kOXunYAUwjLKtclfgf4F\n0sYCr7p7O+DVaLvcKVAkmbtvdPd3ovc7CH8IWiW3VsljZunAucCjya5LspnZocBpRMsBu/vX7r4t\nubVKqlSggZmlAg2BDUmuT6Vy97nAZwWSBwFTovdTgMEVcW4FiirEzDKArsC85NYkqSYBNwD7k12R\nKuBYYAvweNQU96iZNUp2pZLB3T8G7gXWARuB7e7+r+TWqko40t03QvjSCRxRESdRoKgizKwx8Cxw\nrbt/kez6JIOZnQdsdveFya5LFZEKdAMedveuwJdUUNNCVRe1vQ8C2gDfAxqZ2cjk1qr2UKCoAsys\nLiFIZLv7P5JdnyTqBQw0szXAVOAMM3syuVVKqjwgz93z7zCnEwJHbdQPWO3uW9z9G+AfwP9Lcp2q\ngk1m1hIg+rm5Ik6iQJFkZmaENujl7j4h2fVJJncf5+7p7p5B6Kh8zd1r7bdGd/8EWG9mx0dJfYFl\nSaxSMq0DeppZw+j/TF9qacd+ATOBUdH7UcDzFXGS1IooVEqkF/Aj4F0zWxyl3eTus5JYJ6k6fgZk\nm9khwCrg8iTXJyncfZ6ZTQfeITwpuIhaNpWHmT0N9AEON7M84DbgLmCamY0mBNMLK+TcmsJDRETi\nUdOTiIjEpUAhIiJxKVCIiEhcChQiIhKXAoWIiMSlQCEiInEpUIiISFz/HzhKN9SGE6EBAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "epochs = range(1, len(train_acc_list) + 1)\n",
    "\n",
    "plt.plot(epochs, train_acc_list, 'bo', label = 'Trianing acc')\n",
    "plt.plot(epochs, val_acc_list, 'b', label = 'Validation acc')\n",
    "plt.title('Training and validation acc')\n",
    "plt.legend()\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "plt.plot(epochs, train_loss_list, 'bo', label = 'Taining loss')\n",
    "plt.plot(epochs, val_loss_list, 'b', label = 'Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 在测试集上评估模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 读入 checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ResNet(\n",
       "  (conv1): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "  (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (relu): ReLU(inplace=True)\n",
       "  (maxpool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  (layer1): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer2): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(16, 32, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer3): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(32, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(32, 64, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer4): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
       "  (fc): Linear(in_features=128, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读入checkpoint\n",
    "checkpoint = torch.load(\"./model/resnet_9_1_checkpoint.pth\")\n",
    "\n",
    "# 模型读入最优权重\n",
    "model.load_state_dict(checkpoint[\"model\"])\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 在测试集上测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_loss: 0.002003  test_acc:0.837000\n"
     ]
    }
   ],
   "source": [
    "test_loader = DataLoader(test_db, batch_size=BATCH_SIZE, shuffle=False, num_workers=WORKERS, pin_memory=True)\n",
    "\n",
    "test_loss = 0.0\n",
    "correct = 0.0\n",
    "    \n",
    "for batch_X, batch_y in test_loader:\n",
    "    # GPU 可用\n",
    "    if torch.cuda.is_available():\n",
    "        batch_X, batch_y = Variable(batch_X).cuda(), Variable(batch_y).cuda()\n",
    "    else:\n",
    "        batch_X, batch_y = Variable(batch_X), Variable(batch_y)\n",
    "    \n",
    "    out = model(batch_X)\n",
    "    test_loss += criterion(out, batch_y).item()\n",
    "\n",
    "    prediction = out.data.max(1)[1]\n",
    "    correct += float(prediction.eq(batch_y.data).sum().data)\n",
    "\n",
    "test_loss /= len(test_loader.dataset)\n",
    "# 本轮的精度\n",
    "test_acc = correct / len(test_loader.dataset)\n",
    "print(f\"test_loss: {test_loss:2.6f}  test_acc:{test_acc:1.6f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "426.989px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "541px",
    "left": "1485.33px",
    "right": "20px",
    "top": "101.997px",
    "width": "586px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
